package com.coderfans.datacsdn.parse;

import com.coderfans.datagithub.ConsolePipeline;
import com.coderfans.redis.RedisConfig;
import com.coderfans.redis.impl.RedisCacheClient;
import redis.clients.jedis.JedisPool;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Html;

import java.util.List;

/**
 * Created by ${fanchunshuai} on 2016-11-2.
 *
 * @version: V1.0
 * @Desc:
 * @Copyright (c) 2016 58到家-版权所有
 */
public class ParseUserLink implements PageProcessor {
    //步骤一：抓取网站的相关配置 ： 包括编码 抓取间隔 重试次数等
    private Site site = Site.me().setRetryTimes(3).setSleepTime(1000);
    private JedisPool jedisPool = RedisConfig.generatJedisPool();
    private RedisCacheClient setRedis = new RedisCacheClient(jedisPool);

    @Override
    public void process(Page page) {

        page.addTargetRequests(page.getHtml().links().regex("(http://blog.csdn.net/blogdevteam/article/details/74984986)").all());
        //long size = setRedis.sadd(RedisKeyTest2.GITHUB_URL_TEST,"_TEST",page.getUrl().get());
        //System.out.println("siz1e = "+size);
        //System.out.println("url181 = "+page.getUrl().get());
        //System.out.println("url181 = "+page.getHtml().select(new XpathSelector("//a/@href")));
        //*[@id="panel_Category"]/ul[1]/span
        //获取侧边栏数量
        int sideSize = page.getHtml().xpath("//*[@class=\"panel_head\"]/span").all().size();
        //List<String> panelBodyList = page.getHtml().xpath("//*[@class=\"panel_body\"]").all();
//*[@id="panel_Profile"]
        //个人资料
        String sideOne =  page.getHtml().xpath("//div[@id='side']/div[@class='side']/div[@id='panel_Profile']").get();
        /*//博客专栏
        String sideTwo =  page.getHtml().xpath("//div[@id='side']/div[@class='side']/div[@id='panel_Category']").get();*/
        //博客分类+博客专栏
        List<String> sideThree =  page.getHtml().xpath("//div[@id='side']/div[@class='side']//*[@id='panel_Category']").all();
        Html html = page.getHtml();

        for (int i = 0;i < sideSize;i++){
            //侧边栏列表
            String panelTitle =  html.xpath("//div[@id='side']/div[@class='side']/div["+(i+1)+"]/ul[@class='panel_head']/span/text()").get();
            System.out.println("panelTitle = "+panelTitle);
            if(panelTitle.equals("文章分类")){
                List<String> hrefList =  page.getHtml().xpath("//div[@id='side']/div[@class='side']/div["+(i+1)+"]/ul[@class='panel_body']//a/@href").all();

                List<String> titleList =  page.getHtml().xpath("//div[@id='side']/div[@class='side']/div["+(i+1)+"]/ul[@class='panel_body']//a/text()").all();
                System.out.println("a_categoryList 文章分类:"+hrefList);
                System.out.println("a_categoryList 博客专栏-titleList:"+titleList);

            }
            else if(panelTitle.equals("博客专栏")){
                List<String> hrefList =  page.getHtml().xpath("//div[@id='side']/div[@class='side']/div["+(i+1)+"]/ul[@class='panel_body']//a/@href").all();
                List<String> titleList =  page.getHtml().xpath("//div[@id='side']/div[@class='side']/div["+(i+1)+"]/ul[@class='panel_body']//a/text()").all();

                System.out.println("a_serList 博客专栏:"+hrefList);
                System.out.println("a_serList 博客专栏-titleList:"+titleList);

            }
        }





        //自定义侧边栏
        String customSide =  page.getHtml().xpath("//div[@id='side']/div[@class='side']/div[1]").get();
        //System.out.println(customSide);

        //自定义侧边栏
        String customSide2 =  page.getHtml().xpath("//div[@id='side']/div[@class='side']/div[2]").get();
        //System.out.println(customSide2);
        //自定义侧边栏
        String customSide3 =  page.getHtml().xpath("//div[@id='side']/div[@class='side']/div[3]/ul[@class='panel_head']").get();
        //System.out.println(customSide3);
        //获取博客专栏列表的url和名称
        List<String> customSide4 =  page.getHtml().xpath("//div[@id='side']/div[@class='side']/div[4]/ul[@class='panel_body']//a").all();
        //System.out.println(customSide4);
    }

    public Site getSite() {
        return site;
    }

    public static void main(String[] args) {
        Spider.create(new ParseUserLink()).addPipeline(new ConsolePipeline())
                .addUrl("http://blog.csdn.net/blogdevteam/article/details/74984986")
                .thread(2).run();
    }
}
